package engine

import (
	"org/leazen/golearning/crawler/fetcher"
	"log"
)

type SimpleEngine struct {
}

func (e SimpleEngine) Run(seeds  ...Request)  {
	var requests []Request

	//加进请求队列
	for _,r:=range seeds {
		requests = append(requests,r)
	}

	for len(requests) > 0 {
		r:=requests[0]
		requests = requests[1:]

		parseResult, err := e.Worker(r)
		if err!=nil {
			continue
		}

		requests = append(requests,parseResult.Requests ...)

		for _,item:=range parseResult.Items{
			log.Printf("Got Item  %s",item)
		}

	}

}

func (SimpleEngine) Worker(r Request) (ParseResult,error) {


	log.Printf("Fetching %s",r.Url)
	body,err :=fetcher.Fetch(r.Url)
	if err!=nil {
		log.Panicf("Fetch fail  url :  %s  %v",r.Url,err)
		return ParseResult{},err
	}

	return r.ParseFunc(body),nil
}